#include "Configuration.h"
#include "DictProducer.h"
#include "SplitTool.h"
#include <dirent.h>

#include <algorithm>
#include <cstddef>
#include <cstring>
#include <fstream>
#include <iostream>
#include <sstream>
#include <string>
#include <utility>
#include <vector>

using namespace NATSUME;

// 构造函数
EnDictProducer::EnDictProducer(const std::string & dir, SplitTool * pSplitTool) 
    : m_en_dir(dir), m_pSplitTool(pSplitTool)
{
    initColl();
    initEn_Files();
}
// 析构函数
EnDictProducer::~EnDictProducer() { }

// 创建英文字典
void EnDictProducer::buildEn_Dict() {
    for (const auto & filePath : m_en_files) {
        pushEn_Dict(filePath);
    }
    
    for (const auto & elem : m_dictTemporary) {
        m_en_dict.emplace_back(elem);
    }
}
// 将英文词典写入文件
void EnDictProducer::storeEn_Dict(const std::string & filePath) {
    std::ofstream   file(filePath);
    if (! file.good()) {
        std::cerr << "open " << filePath << " error!" << std::endl;
        return;
    }

    for (const auto & elem : m_en_dict) {
        file << elem.first << "\t" << elem.second << "\n";
    }
    
    std::cout << "[success]: DICT for " << filePath << " is stored" << std::endl;
}
// 创建英文索引
void EnDictProducer::buildEn_Index() {
    if (m_en_dict.empty()) {
        std::cerr << " DICT is empty!" << std::endl;
    }

    int             i = 0;
    size_t          charNums, charLen;
    std::string     word, subword;
    for (const auto & elem : m_en_dict) {
        word        = elem.first;
        charNums    = word.size() / GET_BYTENUM_UTF8(word[0]);

        for (size_t idx = 0, n = 0; n != charNums; ++idx, ++n) {
            charLen = GET_BYTENUM_UTF8(word[idx]);
            subword = word.substr(idx, charLen);    // 按照编码格式, 拆分单词

            m_en_index[subword].insert(i);
            idx += (charLen - 1);
        }

        ++i;
    }
}
// 将英文索引写入到文件
void EnDictProducer::storeEn_Index(const std::string & filePath) {
    std::ofstream   file(filePath);
    if (! file.good()) {
        std::cerr << "open " << filePath << " error!" << std::endl;
        return;
    }

    for (const auto & elem : m_en_index) {
        file << elem.first << "\t";
        for (const auto & lineNumbers : elem.second) {
            file << lineNumbers << " "; 
        }
        file << "\n";
    }

    std::cout << "[success]: INDEX for " << filePath << " is stored" << std::endl;
}
// 获取英文索引
std::map<std::string, std::set<int>> &
EnDictProducer::getEn_Index() {
    return m_en_index;
}
// 获取英文文件的绝对路径集合
std::vector<std::string> & 
EnDictProducer::getEn_Files() {
    return m_en_files;
}
// (测试) 查看英文文件路径集合
void EnDictProducer::showEn_Files() const {
    std::cout << "[English files]: " << std::endl;
    PRINT_ELEMENT(m_en_files);
}
// (测试) 查看英文词典
void EnDictProducer::showEn_Dict() const {
    std::cout << "[English Dict]: " << std::endl;
    for (const auto & elem : m_en_dict) {
        std::cout << "[单词]:\t" << elem.first << "\t[词频:]\t" << elem.second << std::endl;
    }
    std::cout << std::endl;
}




// 初始化英文语料库文件的绝对路径集合
void EnDictProducer::initEn_Files() {
    GET_ALLFiles(m_en_dir, m_en_files);
}
// 将filePath文件下的内容, 存储到英文词典中
void EnDictProducer::pushEn_Dict(const std::string & filePath) {
    std::ifstream   file(filePath);
    if (! file.good()) {
        std::cerr << "open " << filePath << " error!" << std::endl;
        return;
    }

    std::string     text, word;          
    while (std::getline(file, text)) {
        STRING_TO_LOWER(text);

        std::vector<std::string>  words(m_pSplitTool->cutEN(text));
        for (auto & word : words) {
            auto it = m_dictTemporary.find(word);
            if (it != m_dictTemporary.end()) {
                ++it->second;
                continue;
            }

             m_dictTemporary.insert(std::make_pair(word, 1));
        }
    }

    std::cout << "[success]: " << filePath << " of EnDict is build" << std::endl;

    file.close();
}
// 初始化容器
void EnDictProducer::initColl() {
    m_dictTemporary.clear();
    m_en_dict.reserve(50000);
    m_en_dict.clear();

    std::cout << "\nNow is using English DICT Producer: " << std::endl;
}







// 构造函数
CnDictProducer::CnDictProducer(const std::string & dir, SplitTool * pSplitTool) 
    : m_cn_dir(dir), m_pSplitTool(pSplitTool)
{
    initColl();
    initCn_Files();
}
// 析构函数
CnDictProducer::~CnDictProducer() { }

// 创建中文字典
void CnDictProducer::buildCn_Dict() {
    for (const auto & filePath : m_cn_files) {
        pushCn_Dict(filePath);
    }
    
    for (const auto & elem : m_dictTemporary) {
        m_cn_dict.emplace_back(elem);
    }
}
// 将中文词典写入文件
void CnDictProducer::storeCn_Dict(const std::string & filePath) {
    std::ofstream   file(filePath);
    if (! file.good()) {
        std::cerr << "open " << filePath << " error!" << std::endl;
        return;
    }

    for (const auto & elem : m_cn_dict) {
        file << elem.first << "\t" << elem.second << "\n";
    }
    
    std::cout << "[success]: DICT for " << filePath << " is stored" << std::endl;
}
// 创建中文索引
void CnDictProducer::buildCn_Index() {
    if (m_cn_dict.empty()) {
        std::cerr << " DICT is empty!" << std::endl;
    }

    int             i = 0;
    size_t          charNums, charLen;
    std::string     word, subword;
    for (const auto & elem : m_cn_dict) {
        word        = elem.first;
        charNums    = word.size() / GET_BYTENUM_UTF8(word[0]);

        for (size_t idx = 0, n = 0; n != charNums; ++idx, ++n) {
            charLen = GET_BYTENUM_UTF8(word[idx]);
            subword = word.substr(idx, charLen);    // 按照编码格式, 拆分单词

            m_cn_index[subword].insert(i);
            idx += (charLen - 1);
        }

        ++i;
    }
}
// 将中文索引写入到文件
void CnDictProducer::storeCn_Index(const std::string & filePath) {
    std::ofstream   file(filePath);
    if (! file.good()) {
        std::cerr << "open " << filePath << " error!" << std::endl;
        return;
    }

    for (const auto & elem : m_cn_index) {
        file << elem.first << "\t";
        for (const auto & lineNumbers : elem.second) {
            file << lineNumbers << " "; 
        }
        file << "\n";
    }

    std::cout << "[success]: INDEX for " << filePath << " is stored" << std::endl;
}
// 获取中文索引
std::map<std::string, std::set<int>> &
CnDictProducer::getCn_Index() {
    return m_cn_index;
}
// 获取中文文件的绝对路径集合
std::vector<std::string> & 
CnDictProducer::getCn_Files() {
    return m_cn_files;
}
// (测试) 查看中文文件路径集合
void CnDictProducer::showCn_Files() const {
    std::cout << "[Chinese files]: " << std::endl;
    PRINT_ELEMENT(m_cn_files);
}
// (测试) 查看中文词典
void CnDictProducer::showCn_Dict() const {
    std::cout << "[Chinese Dict]: " << std::endl;
    for (const auto & elem : m_cn_dict) {
        std::cout << "[单词]:\t" << elem.first << "\t[词频:]\t" << elem.second << std::endl;
    }
    std::cout << std::endl;
}




// 初始化中文语料库文件的绝对路径集合
void CnDictProducer::initCn_Files() {
    GET_ALLFiles(m_cn_dir, m_cn_files);
}
// 将filePath文件下的内容, 存储到中文词典中
void CnDictProducer::pushCn_Dict(const std::string & filePath) {
    std::ifstream   file(filePath);
    if (! file.good()) {
        std::cerr << "open " << filePath << " error!" << std::endl;
        return;
    }

    std::string     text, word;          
    while (std::getline(file, text)) {
        // 中文分词
        std::vector<std::string>  words(m_pSplitTool->cutCN(text));
        for (auto & word : words) {
            auto it = m_dictTemporary.find(word);
            if (it != m_dictTemporary.end()) {
                ++it->second;
                continue;
            }

            m_dictTemporary.insert(std::make_pair(word, 1));
        }

    }

    std::cout << "[success]: " << filePath << " of EnDict is build" << std::endl;

    file.close();
}
// 初始化容器
void CnDictProducer::initColl() {
    m_dictTemporary.clear();
    m_cn_dict.reserve(10000);
    m_cn_dict.clear();
    std::cout << "\nNow is using Chinese DICT Producer: " << std::endl;
}